This is an R Markdown Notebook. When you execute code within the notebook, the results appear beneath the code.

Try executing this chunk by clicking the Run button within the chunk or by placing your cursor inside it and pressing Cmd+Shift+Enter.

# library for LMM 

library(lme4)
Loading required package: Matrix
library(lmerTest)

Attaching package: ‘lmerTest’

The following object is masked from ‘package:lme4’:

    lmer

The following object is masked from ‘package:stats’:

    step
library(car)
Loading required package: carData
df<-read.csv("input/scores_commits.csv", header =TRUE, sep=",")
df <- df[complete.cases(df), ]                  # Apply complete.cases function
df
# convert to nominal factor
df$Group = factor(df$Group)
df$phase = factor(df$phase)
df$log_novelty <- log(df$novelty+1) 
df$log_user_requirement <- log(df$user.requirement+1)
df$log_infovis <- log(df$infovis+1)
df$log_total <- log(df$total+1)
df$log_count <- log(df$count+1)
df$Q7_Q7_1 <- log(df$Q7_Q7_1+1)
df$Q7_Q7_2 <- log(df$Q7_Q7_2+1)
df$Q8_Q8_1 <- log(df$Q8_Q8_1+1)
df$Q10 <- log(df$Q10+1)
# standardizing variables for skills and aspirations. 
cols <- c("Q7_Q7_1", "Q7_Q7_2", "Q8_Q8_1", "Q10", "log_novelty", "log_user_requirement", "log_infovis", "log_total", "log_count")
df[cols] <- scale(df[cols])
df
mod.reduce.novelty <- lmer( log_novelty ~ log_count + Q7_Q7_1 + Q7_Q7_2 + Q8_Q8_1 + Q10 + ( 1 | phase ), data = df, REML = FALSE)
summary(mod.reduce.novelty)
Linear mixed model fit by maximum likelihood . t-tests use Satterthwaite's method ['lmerModLmerTest']
Formula: log_novelty ~ log_count + Q7_Q7_1 + Q7_Q7_2 + Q8_Q8_1 + Q10 +      (1 | phase)
   Data: df

     AIC      BIC   logLik deviance df.resid 
  2103.8   2141.3  -1043.9   2087.8      792 

Scaled residuals: 
    Min      1Q  Median      3Q     Max 
-2.2633 -0.8860  0.1105  0.8440  2.1308 

Random effects:
 Groups   Name        Variance Std.Dev.
 phase    (Intercept) 0.07241  0.2691  
 Residual             0.78242  0.8845  
Number of obs: 800, groups:  phase, 5

Fixed effects:
              Estimate Std. Error         df t value Pr(>|t|)    
(Intercept) -9.165e-15  1.243e-01  4.992e+00   0.000  1.00000    
log_count    3.257e-01  3.179e-02  7.962e+02  10.244  < 2e-16 ***
Q7_Q7_1     -1.935e-01  3.929e-02  7.950e+02  -4.924 1.03e-06 ***
Q7_Q7_2      1.787e-01  4.027e-02  7.950e+02   4.439 1.03e-05 ***
Q8_Q8_1      1.818e-03  3.459e-02  7.950e+02   0.053  0.95811    
Q10          1.014e-01  3.375e-02  7.950e+02   3.004  0.00275 ** 
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Correlation of Fixed Effects:
          (Intr) lg_cnt Q7_Q7_1 Q7_Q7_2 Q8_Q8_
log_count  0.000                              
Q7_Q7_1    0.000  0.092                       
Q7_Q7_2    0.000 -0.050 -0.556                
Q8_Q8_1    0.000 -0.062 -0.085  -0.167        
Q10        0.000 -0.066 -0.021  -0.120  -0.273
AIC(mod.reduce.novelty)
[1] 2103.817
BIC(mod.reduce.novelty)
[1] 2141.294
mod.full.novelty <- lmer( log_novelty ~ factor(Group) + log_count + Q7_Q7_1 + Q7_Q7_2 + Q8_Q8_1 + Q10 + (1 | phase), data = df, REML = FALSE)
summary(mod.full.novelty)
Linear mixed model fit by maximum likelihood . t-tests use Satterthwaite's method ['lmerModLmerTest']
Formula: log_novelty ~ factor(Group) + log_count + Q7_Q7_1 + Q7_Q7_2 +      Q8_Q8_1 + Q10 + (1 | phase)
   Data: df

     AIC      BIC   logLik deviance df.resid 
  2102.5   2154.0  -1040.2   2080.5      789 

Scaled residuals: 
    Min      1Q  Median      3Q     Max 
-2.3873 -0.9114  0.1088  0.8567  2.1940 

Random effects:
 Groups   Name        Variance Std.Dev.
 phase    (Intercept) 0.07219  0.2687  
 Residual             0.77526  0.8805  
Number of obs: 800, groups:  phase, 5

Fixed effects:
                 Estimate Std. Error         df t value Pr(>|t|)    
(Intercept)     -0.127049   0.136604   7.320729  -0.930  0.38199    
factor(Group)1   0.232085   0.090159 794.997658   2.574  0.01023 *  
factor(Group)2   0.167314   0.090050 794.992569   1.858  0.06354 .  
factor(Group)3   0.096403   0.089581 794.994619   1.076  0.28219    
log_count        0.321371   0.031722 796.231055  10.131  < 2e-16 ***
Q7_Q7_1         -0.195842   0.039288 795.004690  -4.985 7.62e-07 ***
Q7_Q7_2          0.176265   0.040328 794.996130   4.371 1.40e-05 ***
Q8_Q8_1         -0.000345   0.034500 794.997224  -0.010  0.99202    
Q10              0.097272   0.034151 794.996547   2.848  0.00451 ** 
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Correlation of Fixed Effects:
            (Intr) fc(G)1 fc(G)2 fc(G)3 lg_cnt Q7_Q7_1 Q7_Q7_2 Q8_Q8_
factr(Grp)1 -0.344                                                   
factr(Grp)2 -0.344  0.512                                            
factr(Grp)3 -0.346  0.526  0.525                                     
log_count    0.019 -0.063 -0.011 -0.041                              
Q7_Q7_1      0.002 -0.046  0.046 -0.012  0.096                       
Q7_Q7_2      0.015  0.008 -0.088 -0.008 -0.053 -0.560                
Q8_Q8_1      0.022 -0.022 -0.050 -0.056 -0.060 -0.086  -0.162        
Q10         -0.005 -0.082  0.095  0.018 -0.056 -0.004  -0.135  -0.274
AIC(mod.full.novelty)
[1] 2102.49
BIC(mod.full.novelty)
[1] 2154.021
anova(mod.full.novelty)
Type III Analysis of Variance Table with Satterthwaite's method
              Sum Sq Mean Sq NumDF  DenDF  F value    Pr(>F)    
factor(Group)  5.706   1.902     3 794.99   2.4534  0.062061 .  
log_count     79.569  79.569     1 796.23 102.6344 < 2.2e-16 ***
Q7_Q7_1       19.264  19.264     1 795.00  24.8486 7.615e-07 ***
Q7_Q7_2       14.810  14.810     1 795.00  19.1037 1.402e-05 ***
Q8_Q8_1        0.000   0.000     1 795.00   0.0001  0.992024    
Q10            6.289   6.289     1 795.00   8.1126  0.004509 ** 
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1
anova(mod.reduce.novelty, mod.full.novelty)
Data: df
Models:
mod.reduce.novelty: log_novelty ~ log_count + Q7_Q7_1 + Q7_Q7_2 + Q8_Q8_1 + Q10 + (1 | phase)
mod.full.novelty: log_novelty ~ factor(Group) + log_count + Q7_Q7_1 + Q7_Q7_2 + Q8_Q8_1 + Q10 + (1 | phase)
                   npar    AIC    BIC  logLik deviance  Chisq Df Pr(>Chisq)  
mod.reduce.novelty    8 2103.8 2141.3 -1043.9   2087.8                       
mod.full.novelty     11 2102.5 2154.0 -1040.2   2080.5 7.3266  3    0.06219 .
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1
library(ALSM)
Loading required package: leaps
Loading required package: SuppDists
step(lm(log_novelty ~ factor(Group) + log_count + Q7_Q7_1 + Q7_Q7_2 + Q8_Q8_1 + Q10, data=df),
method="both", trace = 1 )
Start:  AIC=-115.17
log_novelty ~ factor(Group) + log_count + Q7_Q7_1 + Q7_Q7_2 + 
    Q8_Q8_1 + Q10

                Df Sum of Sq    RSS      AIC
- Q8_Q8_1        1     0.002 677.32 -117.172
<none>                       677.32 -115.173
- factor(Group)  3     5.900 683.22 -114.235
- Q10            1     6.515 683.84 -109.515
- Q7_Q7_2        1    15.138 692.46  -99.490
- Q7_Q7_1        1    19.947 697.27  -93.954
- log_count      1    66.787 744.11  -41.940

Step:  AIC=-117.17
log_novelty ~ factor(Group) + log_count + Q7_Q7_1 + Q7_Q7_2 + 
    Q10

                Df Sum of Sq    RSS      AIC
<none>                       677.32 -117.172
- factor(Group)  3     5.908 683.23 -116.224
- Q10            1     7.106 684.43 -110.823
- Q7_Q7_2        1    15.596 692.92 -100.960
- Q7_Q7_1        1    20.066 697.39  -95.815
- log_count      1    67.067 744.39  -43.638

Call:
lm(formula = log_novelty ~ factor(Group) + log_count + Q7_Q7_1 + 
    Q7_Q7_2 + Q10, data = df)

Coefficients:
   (Intercept)  factor(Group)1  factor(Group)2  factor(Group)3       log_count         Q7_Q7_1         Q7_Q7_2  
      -0.12958         0.23732         0.16843         0.09995         0.29271        -0.19912         0.17849  
           Q10  
       0.09942  
mod.reduce.ur <- lm(log_user_requirement ~ log_count + Q7_Q7_1 + Q7_Q7_2 + Q8_Q8_1 + Q10 , data = df)
summary(mod.reduce.ur)

Call:
lm(formula = log_user_requirement ~ log_count + Q7_Q7_1 + Q7_Q7_2 + 
    Q8_Q8_1 + Q10, data = df)

Residuals:
    Min      1Q  Median      3Q     Max 
-1.9234 -0.9854  0.3754  0.7441  1.6632 

Coefficients:
              Estimate Std. Error t value Pr(>|t|)    
(Intercept) -3.509e-16  3.348e-02   0.000 1.000000    
log_count    2.476e-01  3.383e-02   7.319 6.11e-13 ***
Q7_Q7_1     -1.897e-01  4.207e-02  -4.509 7.51e-06 ***
Q7_Q7_2      1.157e-01  4.311e-02   2.685 0.007412 ** 
Q8_Q8_1     -4.640e-03  3.703e-02  -0.125 0.900327    
Q10          1.194e-01  3.613e-02   3.305 0.000993 ***
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Residual standard error: 0.9471 on 794 degrees of freedom
Multiple R-squared:  0.1086,    Adjusted R-squared:  0.103 
F-statistic: 19.36 on 5 and 794 DF,  p-value: < 2.2e-16
AIC(mod.reduce.ur)
[1] 2191.288
BIC(mod.reduce.ur)
[1] 2224.081
mod.full.ur <- lm(log_user_requirement ~ factor(Group) + log_count + Q7_Q7_1 + Q7_Q7_2 + Q8_Q8_1 + Q10, data = df)
summary(mod.full.ur)

Call:
lm(formula = log_user_requirement ~ factor(Group) + log_count + 
    Q7_Q7_1 + Q7_Q7_2 + Q8_Q8_1 + Q10, data = df)

Residuals:
    Min      1Q  Median      3Q     Max 
-2.0497 -0.9697  0.3109  0.7168  1.7587 

Coefficients:
               Estimate Std. Error t value Pr(>|t|)    
(Intercept)    -0.20771    0.06930  -2.997 0.002809 ** 
factor(Group)1  0.30843    0.09616   3.207 0.001393 ** 
factor(Group)2  0.15234    0.09604   1.586 0.113097    
factor(Group)3  0.34981    0.09554   3.661 0.000268 ***
log_count       0.23922    0.03363   7.114 2.53e-12 ***
Q7_Q7_1        -0.19761    0.04190  -4.716 2.84e-06 ***
Q7_Q7_2         0.11979    0.04301   2.785 0.005482 ** 
Q8_Q8_1        -0.01073    0.03680  -0.292 0.770735    
Q10             0.11094    0.03642   3.046 0.002398 ** 
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Residual standard error: 0.9391 on 791 degrees of freedom
Multiple R-squared:  0.1269,    Adjusted R-squared:  0.1181 
F-statistic: 14.37 on 8 and 791 DF,  p-value: < 2.2e-16
AIC(mod.full.ur)
[1] 2180.715
BIC(mod.full.ur)
[1] 2227.561
anova(mod.reduce.ur, mod.full.ur)
Analysis of Variance Table

Model 1: log_user_requirement ~ log_count + Q7_Q7_1 + Q7_Q7_2 + Q8_Q8_1 + 
    Q10
Model 2: log_user_requirement ~ factor(Group) + log_count + Q7_Q7_1 + 
    Q7_Q7_2 + Q8_Q8_1 + Q10
  Res.Df    RSS Df Sum of Sq      F    Pr(>F)    
1    794 712.19                                  
2    791 697.59  3    14.602 5.5192 0.0009401 ***
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1
library(ALSM)
step(lm(log_user_requirement ~ factor(Group) + log_count + Q7_Q7_1 + Q7_Q7_2 + Q8_Q8_1 + Q10, data=df),
method="both", trace = 1 )
Start:  AIC=-91.59
log_user_requirement ~ factor(Group) + log_count + Q7_Q7_1 + 
    Q7_Q7_2 + Q8_Q8_1 + Q10

                Df Sum of Sq    RSS     AIC
- Q8_Q8_1        1     0.075 697.66 -93.500
<none>                       697.59 -91.586
- Q7_Q7_2        1     6.840 704.43 -85.780
- Q10            1     8.181 705.77 -84.258
- factor(Group)  3    14.602 712.19 -81.013
- Q7_Q7_1        1    19.617 717.20 -71.400
- log_count      1    44.633 742.22 -43.971

Step:  AIC=-93.5
log_user_requirement ~ factor(Group) + log_count + Q7_Q7_1 + 
    Q7_Q7_2 + Q10

                Df Sum of Sq    RSS     AIC
<none>                       697.66 -93.500
- Q7_Q7_2        1     6.788 704.45 -87.755
- Q10            1     8.389 706.05 -85.938
- factor(Group)  3    14.541 712.20 -82.997
- Q7_Q7_1        1    19.975 717.64 -72.917
- log_count      1    44.575 742.24 -45.954

Call:
lm(formula = log_user_requirement ~ factor(Group) + log_count + 
    Q7_Q7_1 + Q7_Q7_2 + Q10, data = df)

Coefficients:
   (Intercept)  factor(Group)1  factor(Group)2  factor(Group)3       log_count         Q7_Q7_1         Q7_Q7_2  
       -0.2068          0.3078          0.1509          0.3482          0.2386         -0.1987          0.1178  
           Q10  
        0.1080  
mod.reduce.vis <- lmer( log_infovis ~ log_count + Q7_Q7_1 + Q7_Q7_2 + Q8_Q8_1 + Q10 + ( 1 | phase) , data = df, REML = FALSE)
summary(mod.reduce.vis)
Linear mixed model fit by maximum likelihood . t-tests use Satterthwaite's method ['lmerModLmerTest']
Formula: log_infovis ~ log_count + Q7_Q7_1 + Q7_Q7_2 + Q8_Q8_1 + Q10 +      (1 | phase)
   Data: df

     AIC      BIC   logLik deviance df.resid 
  2124.2   2161.7  -1054.1   2108.2      792 

Scaled residuals: 
    Min      1Q  Median      3Q     Max 
-3.2613 -0.1514  0.3040  0.6139  1.7276 

Random effects:
 Groups   Name        Variance Std.Dev.
 phase    (Intercept) 0.04898  0.2213  
 Residual             0.80459  0.8970  
Number of obs: 800, groups:  phase, 5

Fixed effects:
              Estimate Std. Error         df t value Pr(>|t|)    
(Intercept) -3.387e-15  1.039e-01  4.990e+00   0.000  1.00000    
log_count    3.267e-01  3.223e-02  7.968e+02  10.137  < 2e-16 ***
Q7_Q7_1     -1.748e-01  3.984e-02  7.950e+02  -4.386 1.31e-05 ***
Q7_Q7_2      2.028e-01  4.083e-02  7.950e+02   4.965 8.39e-07 ***
Q8_Q8_1     -7.557e-02  3.507e-02  7.950e+02  -2.155  0.03149 *  
Q10          1.007e-01  3.422e-02  7.950e+02   2.944  0.00334 ** 
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Correlation of Fixed Effects:
          (Intr) lg_cnt Q7_Q7_1 Q7_Q7_2 Q8_Q8_
log_count  0.000                              
Q7_Q7_1    0.000  0.092                       
Q7_Q7_2    0.000 -0.050 -0.556                
Q8_Q8_1    0.000 -0.062 -0.085  -0.167        
Q10        0.000 -0.066 -0.021  -0.120  -0.273
AIC(mod.reduce.vis)
[1] 2124.233
BIC(mod.reduce.vis)
[1] 2161.709
mod.full.vis <- lmer( log_infovis ~ factor(Group) + log_count + Q7_Q7_1 + Q7_Q7_2 + Q8_Q8_1 + Q10 + ( 1 | phase) , data = df, REML = FALSE)
summary(mod.full.vis)
Linear mixed model fit by maximum likelihood . t-tests use Satterthwaite's method ['lmerModLmerTest']
Formula: log_infovis ~ factor(Group) + log_count + Q7_Q7_1 + Q7_Q7_2 +      Q8_Q8_1 + Q10 + (1 | phase)
   Data: df

     AIC      BIC   logLik deviance df.resid 
  2107.2   2158.8  -1042.6   2085.2      789 

Scaled residuals: 
    Min      1Q  Median      3Q     Max 
-3.4115 -0.1553  0.3094  0.6174  1.6554 

Random effects:
 Groups   Name        Variance Std.Dev.
 phase    (Intercept) 0.04873  0.2207  
 Residual             0.78168  0.8841  
Number of obs: 800, groups:  phase, 5

Fixed effects:
                Estimate Std. Error        df t value Pr(>|t|)    
(Intercept)     -0.27038    0.11833   8.50468  -2.285 0.049801 *  
factor(Group)1   0.36811    0.09053 794.99787   4.066 5.26e-05 ***
factor(Group)2   0.30044    0.09042 794.99050   3.323 0.000932 ***
factor(Group)3   0.38658    0.08995 794.99347   4.298 1.94e-05 ***
log_count        0.31844    0.03185 796.72863   9.999  < 2e-16 ***
Q7_Q7_1         -0.17878    0.03945 795.00805  -4.532 6.74e-06 ***
Q7_Q7_2          0.19995    0.04049 794.99566   4.938 9.64e-07 ***
Q8_Q8_1         -0.08388    0.03464 794.99724  -2.421 0.015682 *  
Q10              0.09844    0.03429 794.99626   2.871 0.004207 ** 
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Correlation of Fixed Effects:
            (Intr) fc(G)1 fc(G)2 fc(G)3 lg_cnt Q7_Q7_1 Q7_Q7_2 Q8_Q8_
factr(Grp)1 -0.399                                                   
factr(Grp)2 -0.398  0.512                                            
factr(Grp)3 -0.401  0.526  0.525                                     
log_count    0.022 -0.063 -0.011 -0.041                              
Q7_Q7_1      0.002 -0.046  0.046 -0.012  0.096                       
Q7_Q7_2      0.017  0.008 -0.088 -0.008 -0.053 -0.560                
Q8_Q8_1      0.025 -0.022 -0.050 -0.056 -0.060 -0.086  -0.162        
Q10         -0.006 -0.082  0.095  0.018 -0.056 -0.004  -0.135  -0.274
AIC(mod.full.vis)
[1] 2107.234
BIC(mod.full.vis)
[1] 2158.764
anova(mod.reduce.vis, mod.full.vis)
Data: df
Models:
mod.reduce.vis: log_infovis ~ log_count + Q7_Q7_1 + Q7_Q7_2 + Q8_Q8_1 + Q10 + (1 | phase)
mod.full.vis: log_infovis ~ factor(Group) + log_count + Q7_Q7_1 + Q7_Q7_2 + Q8_Q8_1 + Q10 + (1 | phase)
               npar    AIC    BIC  logLik deviance  Chisq Df Pr(>Chisq)    
mod.reduce.vis    8 2124.2 2161.7 -1054.1   2108.2                         
mod.full.vis     11 2107.2 2158.8 -1042.6   2085.2 22.999  3   4.04e-05 ***
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1
library(ALSM)
step(lm(log_infovis ~ Group + log_count + Q7_Q7_1 + Q7_Q7_2 + Q8_Q8_1 + Q10, data=df),
method="both", trace = 1 )
Start:  AIC=-131.12
log_infovis ~ Group + log_count + Q7_Q7_1 + Q7_Q7_2 + Q8_Q8_1 + 
    Q10

            Df Sum of Sq    RSS      AIC
<none>                   663.96 -131.117
- Q8_Q8_1    1     4.428 668.38 -127.799
- Q10        1     6.615 670.57 -125.186
- Group      3    18.523 682.48 -115.104
- Q7_Q7_1    1    16.528 680.48 -113.446
- Q7_Q7_2    1    19.340 683.30 -110.147
- log_count  1    68.596 732.55  -54.463

Call:
lm(formula = log_infovis ~ Group + log_count + Q7_Q7_1 + Q7_Q7_2 + 
    Q8_Q8_1 + Q10, data = df)

Coefficients:
(Intercept)       Group1       Group2       Group3    log_count      Q7_Q7_1      Q7_Q7_2      Q8_Q8_1  
   -0.27221      0.37203      0.30113      0.38910      0.29656     -0.18139      0.20142     -0.08245  
        Q10  
    0.09975  
mod.reduce.total <- lm( log_total ~ log_count + Q7_Q7_1 + Q7_Q7_2 + Q8_Q8_1 + Q10 , data = df)
summary(mod.reduce.total)

Call:
lm(formula = log_total ~ log_count + Q7_Q7_1 + Q7_Q7_2 + Q8_Q8_1 + 
    Q10, data = df)

Residuals:
    Min      1Q  Median      3Q     Max 
-3.1861 -0.1993  0.2443  0.5703  1.4738 

Coefficients:
              Estimate Std. Error t value Pr(>|t|)    
(Intercept) -1.422e-16  3.222e-02   0.000 1.000000    
log_count    3.419e-01  3.255e-02  10.503  < 2e-16 ***
Q7_Q7_1     -1.852e-01  4.048e-02  -4.576 5.51e-06 ***
Q7_Q7_2      1.875e-01  4.148e-02   4.520 7.12e-06 ***
Q8_Q8_1     -8.656e-02  3.563e-02  -2.429 0.015349 *  
Q10          1.241e-01  3.477e-02   3.570 0.000378 ***
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Residual standard error: 0.9113 on 794 degrees of freedom
Multiple R-squared:  0.1748,    Adjusted R-squared:  0.1696 
F-statistic: 33.63 on 5 and 794 DF,  p-value: < 2.2e-16
AIC(mod.reduce.total)
[1] 2129.645
BIC(mod.reduce.total)
[1] 2162.437
mod.full.total <- lm( log_total ~ Group + Q7_Q7_1 + Q7_Q7_2 + Q8_Q8_1 + Q10 , data = df)
summary(mod.full.total)

Call:
lm(formula = log_total ~ Group + Q7_Q7_1 + Q7_Q7_2 + Q8_Q8_1 + 
    Q10, data = df)

Residuals:
    Min      1Q  Median      3Q     Max 
-2.9973 -0.1279  0.2773  0.5482  1.4041 

Coefficients:
            Estimate Std. Error t value Pr(>|t|)    
(Intercept) -0.29834    0.07061  -4.225 2.66e-05 ***
Group1       0.43251    0.09786   4.420 1.13e-05 ***
Group2       0.28093    0.09793   2.869 0.004231 ** 
Group3       0.45080    0.09734   4.631 4.25e-06 ***
Q7_Q7_1     -0.23066    0.04253  -5.424 7.76e-08 ***
Q7_Q7_2      0.20946    0.04380   4.783 2.06e-06 ***
Q8_Q8_1     -0.07307    0.03745  -1.951 0.051416 .  
Q10          0.13970    0.03708   3.767 0.000177 ***
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Residual standard error: 0.9576 on 792 degrees of freedom
Multiple R-squared:  0.09105,   Adjusted R-squared:  0.08302 
F-statistic: 11.33 on 7 and 792 DF,  p-value: 9.643e-14
AIC(mod.full.total)
[1] 2210.925
BIC(mod.full.total)
[1] 2253.087
anova(mod.reduce.total, mod.full.total)
Analysis of Variance Table

Model 1: log_total ~ log_count + Q7_Q7_1 + Q7_Q7_2 + Q8_Q8_1 + Q10
Model 2: log_total ~ Group + Q7_Q7_1 + Q7_Q7_2 + Q8_Q8_1 + Q10
  Res.Df    RSS Df Sum of Sq F Pr(>F)
1    794 659.37                      
2    792 726.25  2   -66.874         
# convert to nominal factor
df$Group = factor(df$Group)
df$phase = factor(df$phase)
library(plyr)
ddply(df, ~ Group * phase, function(data) summary(data$log_novelty) )
ddply(df, ~ Group * phase, summarise, log_novelty.mean=mean(log_novelty), log_novelty.sd = sd(log_novelty))
# histograms for two factors
boxplot(log_novelty ~ Group * phase, data = df, xlab="Group.Phase", ylab="log_novelty")

with(df, interaction.plot(Group, phase, log_novelty, ylim=c(0, max(log_novelty)))) # interaction plot

# histograms for two factors
boxplot(log_novelty ~ Group * phase, data = df, xlab="Group.Phase", ylab="log_user_requirement")

with(df, interaction.plot(Group, phase, log_user_requirement, ylim=c(0, max(log_user_requirement)))) # interaction plot

# histograms for two factors
boxplot(log_novelty ~ Group * phase, data = df, xlab="Group.Phase", ylab="log_infovis")

with(df, interaction.plot(Group, phase, log_infovis, ylim=c(0, max(log_infovis)))) # interaction plot

# histograms for two factors
boxplot(log_novelty ~ Group * phase, data = df, xlab="Group.Phase", ylab="log_total")

with(df, interaction.plot(Group, phase, log_total, ylim=c(0, max(log_total)))) # interaction plot

m = lmer(log_novelty ~ Group + (1|Student), data=df, REML=FALSE)
summary(m)
Linear mixed model fit by maximum likelihood . t-tests use Satterthwaite's method ['lmerModLmerTest']
Formula: log_novelty ~ Group + (1 | Student)
   Data: df

     AIC      BIC   logLik deviance df.resid 
  2046.0   2074.1  -1017.0   2034.0      794 

Scaled residuals: 
    Min      1Q  Median      3Q     Max 
-2.2669 -0.6407  0.1015  0.6356  2.3143 

Random effects:
 Groups   Name        Variance Std.Dev.
 Student  (Intercept) 0.4473   0.6688  
 Residual             0.5366   0.7325  
Number of obs: 800, groups:  Student, 159

Fixed effects:
            Estimate Std. Error       df t value Pr(>|t|)  
(Intercept)  -0.1667     0.1224 159.4644  -1.361    0.175  
Group1        0.2933     0.1698 159.0131   1.728    0.086 .
Group2        0.1952     0.1689 159.4644   1.156    0.249  
Group3        0.1401     0.1689 159.4644   0.829    0.408  
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Correlation of Fixed Effects:
       (Intr) Group1 Group2
Group1 -0.721              
Group2 -0.725  0.523       
Group3 -0.725  0.523  0.526
plot(resid(m, type = "pearson") ~ fitted(m))

qqnorm(resid(m, type = "pearson"))
qqline(resid(m, type = "pearson"))

# library for LMM we will use on relational log_novelty 

library(lme4)
library(lmerTest)
library(car)

set sum-to-zero contrast for ANOVA cells

contrasts(df$Group) <= "contr.sum"
     1    2    3
0 TRUE TRUE TRUE
1 TRUE TRUE TRUE
2 TRUE TRUE TRUE
3 TRUE TRUE TRUE
contrasts(df$phase) <= "contr.sum"
     2    3    4    5
1 TRUE TRUE TRUE TRUE
2 TRUE TRUE TRUE TRUE
3 TRUE TRUE TRUE TRUE
4 TRUE TRUE TRUE TRUE
5 TRUE TRUE TRUE TRUE
# phase is nested within group 
fit <- lm(log_total ~ log_count + Q7_Q7_1 + Q7_Q7_2 + Q8_Q8_1 + Q10 , data = df)
summary(fit)

Call:
lm(formula = log_total ~ log_count + Q7_Q7_1 + Q7_Q7_2 + Q8_Q8_1 + 
    Q10, data = df)

Residuals:
    Min      1Q  Median      3Q     Max 
-3.1861 -0.1993  0.2443  0.5703  1.4738 

Coefficients:
              Estimate Std. Error t value Pr(>|t|)    
(Intercept) -1.422e-16  3.222e-02   0.000 1.000000    
log_count    3.419e-01  3.255e-02  10.503  < 2e-16 ***
Q7_Q7_1     -1.852e-01  4.048e-02  -4.576 5.51e-06 ***
Q7_Q7_2      1.875e-01  4.148e-02   4.520 7.12e-06 ***
Q8_Q8_1     -8.656e-02  3.563e-02  -2.429 0.015349 *  
Q10          1.241e-01  3.477e-02   3.570 0.000378 ***
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Residual standard error: 0.9113 on 794 degrees of freedom
Multiple R-squared:  0.1748,    Adjusted R-squared:  0.1696 
F-statistic: 33.63 on 5 and 794 DF,  p-value: < 2.2e-16
library(multcomp)
Loading required package: mvtnorm
Loading required package: survival
Loading required package: TH.data
Loading required package: MASS

Attaching package: ‘TH.data’

The following object is masked from ‘package:MASS’:

    geyser
library(lsmeans)
Loading required package: emmeans
The 'lsmeans' package is now basically a front end for 'emmeans'.
Users are encouraged to switch the rest of the way.
See help('transition') for more information, including how to
convert old 'lsmeans' objects and scripts to work with 'emmeans'.
#summary(glht(fit, lsm(pairwise ~ roup / phase)), test = adjusted(type='holm'))
fit.full <- lm(log_total ~ Group  + log_count + Q7_Q7_1 + Q7_Q7_2 + Q8_Q8_1 + Q10, data = df)
summary(fit.full)

Call:
lm(formula = log_total ~ Group + log_count + Q7_Q7_1 + Q7_Q7_2 + 
    Q8_Q8_1 + Q10, data = df)

Residuals:
    Min      1Q  Median      3Q     Max 
-3.1191 -0.2201  0.2313  0.5645  1.3650 

Coefficients:
            Estimate Std. Error t value Pr(>|t|)    
(Intercept) -0.27052    0.06637  -4.076 5.04e-05 ***
Group1       0.37292    0.09209   4.050 5.64e-05 ***
Group2       0.27040    0.09198   2.940 0.003380 ** 
Group3       0.41236    0.09150   4.507 7.58e-06 ***
log_count    0.33294    0.03220  10.339  < 2e-16 ***
Q7_Q7_1     -0.19094    0.04013  -4.758 2.32e-06 ***
Q7_Q7_2      0.18702    0.04119   4.540 6.49e-06 ***
Q8_Q8_1     -0.09491    0.03524  -2.693 0.007223 ** 
Q10          0.11965    0.03488   3.430 0.000635 ***
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Residual standard error: 0.8994 on 791 degrees of freedom
Multiple R-squared:  0.1993,    Adjusted R-squared:  0.1912 
F-statistic: 24.61 on 8 and 791 DF,  p-value: < 2.2e-16
anova(fit, fit.full)
Analysis of Variance Table

Model 1: log_total ~ log_count + Q7_Q7_1 + Q7_Q7_2 + Q8_Q8_1 + Q10
Model 2: log_total ~ Group + log_count + Q7_Q7_1 + Q7_Q7_2 + Q8_Q8_1 + 
    Q10
  Res.Df    RSS Df Sum of Sq      F    Pr(>F)    
1    794 659.37                                  
2    791 639.79  3    19.585 8.0715 2.664e-05 ***
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1
# histograms for two factors
boxplot(log_total ~ Group * phase, data = df, xlab="Group.Phase", ylab="log_total")

with(df, interaction.plot(Group, phase, log_total, ylim=c(0, max(log_total)))) # interaction plot

fit.lmer <- lmer(log_total ~ (1 | Group) + (1| phase:Group), data = df, REML= FALSE)
summary(fit.lmer)
Linear mixed model fit by maximum likelihood . t-tests use Satterthwaite's method ['lmerModLmerTest']
Formula: log_total ~ (1 | Group) + (1 | phase:Group)
   Data: df

     AIC      BIC   logLik deviance df.resid 
  2254.0   2272.7  -1123.0   2246.0      796 

Scaled residuals: 
     Min       1Q   Median       3Q      Max 
-2.94437 -0.01942  0.28792  0.61016  1.43336 

Random effects:
 Groups      Name        Variance Std.Dev.
 phase:Group (Intercept) 0.03675  0.1917  
 Group       (Intercept) 0.02060  0.1435  
 Residual                0.94281  0.9710  
Number of obs: 800, groups:  phase:Group, 20; Group, 4

Fixed effects:
             Estimate Std. Error        df t value Pr(>|t|)
(Intercept) -0.006239   0.090377  3.950735  -0.069    0.948
library(lmerTest)
fit.lmer <- lmer(log_total ~ Group  + log_count + Q7_Q7_1 + Q7_Q7_2 + Q8_Q8_1 + Q10 + ( 1 | phase), data = df, REML= FALSE)
summary(fit.lmer)
Linear mixed model fit by maximum likelihood . t-tests use Satterthwaite's method ['lmerModLmerTest']
Formula: log_total ~ Group + log_count + Q7_Q7_1 + Q7_Q7_2 + Q8_Q8_1 +      Q10 + (1 | phase)
   Data: df

     AIC      BIC   logLik deviance df.resid 
  2071.8   2123.3  -1024.9   2049.8      789 

Scaled residuals: 
    Min      1Q  Median      3Q     Max 
-3.7050 -0.2260  0.2540  0.5958  1.8126 

Random effects:
 Groups   Name        Variance Std.Dev.
 phase    (Intercept) 0.05292  0.2300  
 Residual             0.74729  0.8645  
Number of obs: 800, groups:  phase, 5

Fixed effects:
             Estimate Std. Error        df t value Pr(>|t|)    
(Intercept)  -0.26867    0.12105   8.07626  -2.219 0.056926 .  
Group1        0.36895    0.08852 794.99855   4.168 3.41e-05 ***
Group2        0.26970    0.08841 794.99199   3.051 0.002360 ** 
Group3        0.40980    0.08795 794.99463   4.659 3.72e-06 ***
log_count     0.35508    0.03114 796.55633  11.402  < 2e-16 ***
Q7_Q7_1      -0.18830    0.03857 795.00761  -4.882 1.27e-06 ***
Q7_Q7_2       0.18553    0.03959 794.99658   4.686 3.28e-06 ***
Q8_Q8_1      -0.09636    0.03387 794.99799  -2.845 0.004556 ** 
Q10           0.11832    0.03353 794.99712   3.529 0.000441 ***
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Correlation of Fixed Effects:
          (Intr) Group1 Group2 Group3 lg_cnt Q7_Q7_1 Q7_Q7_2 Q8_Q8_
Group1    -0.381                                                   
Group2    -0.381  0.512                                            
Group3    -0.383  0.526  0.525                                     
log_count  0.021 -0.063 -0.011 -0.041                              
Q7_Q7_1    0.002 -0.046  0.046 -0.012  0.096                       
Q7_Q7_2    0.017  0.008 -0.088 -0.008 -0.053 -0.560                
Q8_Q8_1    0.024 -0.022 -0.050 -0.056 -0.060 -0.086  -0.162        
Q10       -0.006 -0.082  0.095  0.018 -0.056 -0.004  -0.135  -0.274
anova(fit.lmer)
Type III Analysis of Variance Table with Satterthwaite's method
          Sum Sq Mean Sq NumDF  DenDF  F value    Pr(>F)    
Group     19.278   6.426     3 794.99   8.5989 1.272e-05 ***
log_count 97.157  97.157     1 796.56 130.0129 < 2.2e-16 ***
Q7_Q7_1   17.809  17.809     1 795.01  23.8318 1.271e-06 ***
Q7_Q7_2   16.407  16.407     1 795.00  21.9558 3.281e-06 ***
Q8_Q8_1    6.048   6.048     1 795.00   8.0936 0.0045564 ** 
Q10        9.306   9.306     1 795.00  12.4524 0.0004414 ***
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1
library(multcomp)
summary(glht(fit.lmer, linfct = mcp(Group = "Tukey")), test = adjusted("holm"))

     Simultaneous Tests for General Linear Hypotheses

Multiple Comparisons of Means: Tukey Contrasts


Fit: lmer(formula = log_total ~ Group + log_count + Q7_Q7_1 + Q7_Q7_2 + 
    Q8_Q8_1 + Q10 + (1 | phase), data = df, REML = FALSE)

Linear Hypotheses:
           Estimate Std. Error z value Pr(>|z|)    
1 - 0 == 0  0.36895    0.08852   4.168 0.000154 ***
2 - 0 == 0  0.26970    0.08841   3.051 0.009136 ** 
3 - 0 == 0  0.40980    0.08795   4.659  1.9e-05 ***
2 - 1 == 0 -0.09925    0.08739  -1.136 0.512120    
3 - 1 == 0  0.04085    0.08594   0.475 0.634555    
3 - 2 == 0  0.14010    0.08597   1.630 0.309475    
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1
(Adjusted p values reported -- holm method)
fit.lmer.reduced <- lmer(log_total ~ log_count + Q7_Q7_1 + Q7_Q7_2 + Q8_Q8_1 + Q10 + ( 1 | phase), data = df, REML=FALSE)
summary(fit.lmer.reduced)
Linear mixed model fit by maximum likelihood . t-tests use Satterthwaite's method ['lmerModLmerTest']
Formula: log_total ~ log_count + Q7_Q7_1 + Q7_Q7_2 + Q8_Q8_1 + Q10 + (1 |      phase)
   Data: df

     AIC      BIC   logLik deviance df.resid 
  2091.2   2128.7  -1037.6   2075.2      792 

Scaled residuals: 
    Min      1Q  Median      3Q     Max 
-3.4850 -0.2298  0.2612  0.5992  1.8832 

Random effects:
 Groups   Name        Variance Std.Dev.
 phase    (Intercept) 0.05319  0.2306  
 Residual             0.77150  0.8784  
Number of obs: 800, groups:  phase, 5

Fixed effects:
              Estimate Std. Error         df t value Pr(>|t|)    
(Intercept) -8.153e-16  1.077e-01  4.992e+00   0.000 1.000000    
log_count    3.640e-01  3.156e-02  7.966e+02  11.532  < 2e-16 ***
Q7_Q7_1     -1.827e-01  3.902e-02  7.950e+02  -4.683 3.33e-06 ***
Q7_Q7_2      1.861e-01  3.998e-02  7.950e+02   4.655 3.80e-06 ***
Q8_Q8_1     -8.804e-02  3.435e-02  7.950e+02  -2.563 0.010547 *  
Q10          1.226e-01  3.351e-02  7.950e+02   3.658 0.000271 ***
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Correlation of Fixed Effects:
          (Intr) lg_cnt Q7_Q7_1 Q7_Q7_2 Q8_Q8_
log_count  0.000                              
Q7_Q7_1    0.000  0.092                       
Q7_Q7_2    0.000 -0.050 -0.556                
Q8_Q8_1    0.000 -0.062 -0.085  -0.167        
Q10        0.000 -0.066 -0.021  -0.120  -0.273
anova(fit.lmer.reduced, fit.lmer)
Data: df
Models:
fit.lmer.reduced: log_total ~ log_count + Q7_Q7_1 + Q7_Q7_2 + Q8_Q8_1 + Q10 + (1 | phase)
fit.lmer: log_total ~ Group + log_count + Q7_Q7_1 + Q7_Q7_2 + Q8_Q8_1 + Q10 + (1 | phase)
                 npar    AIC    BIC  logLik deviance  Chisq Df Pr(>Chisq)    
fit.lmer.reduced    8 2091.2 2128.7 -1037.6   2075.2                         
fit.lmer           11 2071.8 2123.3 -1024.9   2049.8 25.388  3  1.281e-05 ***
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

User Requirement Score

# phase is nested within group 
fit.requirement.full <- lmer(log_user_requirement ~  factor(Group) + log_count + Q7_Q7_1 + Q7_Q7_2 + Q8_Q8_1 + Q10 + ( 1 | phase), data = df, REML = FALSE)
Anova(fit.requirement.full, type=3, test.statistics="F")
Analysis of Deviance Table (Type III Wald chisquare tests)

Response: log_user_requirement
                Chisq Df Pr(>Chisq)    
(Intercept)    2.5959  1  0.1071398    
factor(Group) 17.5649  3  0.0005407 ***
log_count     65.8539  1  4.856e-16 ***
Q7_Q7_1       23.4133  1  1.307e-06 ***
Q7_Q7_2        8.1810  1  0.0042331 ** 
Q8_Q8_1        0.1217  1  0.7271747    
Q10            9.7952  1  0.0017497 ** 
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1
fit.requirement.full
Linear mixed model fit by maximum likelihood  ['lmerModLmerTest']
Formula: log_user_requirement ~ factor(Group) + log_count + Q7_Q7_1 +  
    Q7_Q7_2 + Q8_Q8_1 + Q10 + (1 | phase)
   Data: df
      AIC       BIC    logLik  deviance  df.resid 
 2139.609  2191.140 -1058.805  2117.609       789 
Random effects:
 Groups   Name        Std.Dev.
 phase    (Intercept) 0.2436  
 Residual             0.9018  
Number of obs: 800, groups:  phase, 5
Fixed Effects:
   (Intercept)  factor(Group)1  factor(Group)2  factor(Group)3       log_count         Q7_Q7_1         Q7_Q7_2  
      -0.20567         0.30406         0.15157         0.34699         0.26363        -0.19470         0.11814  
       Q8_Q8_1             Q10  
      -0.01233         0.10947  
# histograms for two factors
boxplot(log_user_requirement ~ Group * phase, data = df, xlab="Group.Phase", ylab="log_user_requirement")

with(df, interaction.plot(Group, phase, log_user_requirement, ylim=c(0, max(log_user_requirement)))) # interaction plot

# phase is nested within group 
fit.requirement <- lmer(log_user_requirement ~  log_count + Q7_Q7_1 + Q7_Q7_2 + Q8_Q8_1 + Q10 +  ( 1 | phase), data = df, REML = FALSE)
Anova(fit, type=3, test.statistics="F")
Anova Table (Type III tests)

Response: log_total
            Sum Sq  Df  F value    Pr(>F)    
(Intercept)   0.00   1   0.0000  1.000000    
log_count    91.62   1 110.3216 < 2.2e-16 ***
Q7_Q7_1      17.39   1  20.9366 5.507e-06 ***
Q7_Q7_2      16.97   1  20.4327 7.117e-06 ***
Q8_Q8_1       4.90   1   5.9015  0.015349 *  
Q10          10.59   1  12.7473  0.000378 ***
Residuals   659.37 794                       
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1
fit.requirement
Linear mixed model fit by maximum likelihood  ['lmerModLmerTest']
Formula: log_user_requirement ~ log_count + Q7_Q7_1 + Q7_Q7_2 + Q8_Q8_1 +      Q10 + (1 | phase)
   Data: df
      AIC       BIC    logLik  deviance  df.resid 
 2150.984  2188.461 -1067.492  2134.984       792 
Random effects:
 Groups   Name        Std.Dev.
 phase    (Intercept) 0.2442  
 Residual             0.9117  
Number of obs: 800, groups:  phase, 5
Fixed Effects:
(Intercept)    log_count      Q7_Q7_1      Q7_Q7_2      Q8_Q8_1          Q10  
 -1.007e-14    2.720e-01   -1.869e-01    1.142e-01   -6.273e-03    1.177e-01  
plot(resid(m, type = "pearson") ~ fitted(m))

qqnorm(resid(m, type = "pearson"))
qqline(resid(m, type = "pearson"))

anova(fit.requirement, fit.requirement.full)
Data: df
Models:
fit.requirement: log_user_requirement ~ log_count + Q7_Q7_1 + Q7_Q7_2 + Q8_Q8_1 + Q10 + (1 | phase)
fit.requirement.full: log_user_requirement ~ factor(Group) + log_count + Q7_Q7_1 + Q7_Q7_2 + Q8_Q8_1 + Q10 + (1 | phase)
                     npar    AIC    BIC  logLik deviance  Chisq Df Pr(>Chisq)    
fit.requirement         8 2151.0 2188.5 -1067.5   2135.0                         
fit.requirement.full   11 2139.6 2191.1 -1058.8   2117.6 17.374  3  0.0005919 ***
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1
LS0tCnRpdGxlOiAiUiBOb3RlYm9vayIKb3V0cHV0OgogIHBkZl9kb2N1bWVudDogZGVmYXVsdAogIGh0bWxfbm90ZWJvb2s6IGRlZmF1bHQKLS0tCgpUaGlzIGlzIGFuIFtSIE1hcmtkb3duXShodHRwOi8vcm1hcmtkb3duLnJzdHVkaW8uY29tKSBOb3RlYm9vay4gV2hlbiB5b3UgZXhlY3V0ZSBjb2RlIHdpdGhpbiB0aGUgbm90ZWJvb2ssIHRoZSByZXN1bHRzIGFwcGVhciBiZW5lYXRoIHRoZSBjb2RlLiAKClRyeSBleGVjdXRpbmcgdGhpcyBjaHVuayBieSBjbGlja2luZyB0aGUgKlJ1biogYnV0dG9uIHdpdGhpbiB0aGUgY2h1bmsgb3IgYnkgcGxhY2luZyB5b3VyIGN1cnNvciBpbnNpZGUgaXQgYW5kIHByZXNzaW5nICpDbWQrU2hpZnQrRW50ZXIqLiAKCmBgYHtyfQojIGxpYnJhcnkgZm9yIExNTSAKCmxpYnJhcnkobG1lNCkKbGlicmFyeShsbWVyVGVzdCkKbGlicmFyeShjYXIpCmBgYAoKCgpgYGB7cn0KZGY8LXJlYWQuY3N2KCJpbnB1dC9zY29yZXNfY29tbWl0cy5jc3YiLCBoZWFkZXIgPVRSVUUsIHNlcD0iLCIpCmRmIDwtIGRmW2NvbXBsZXRlLmNhc2VzKGRmKSwgXSAgICAgICAgICAgICAgICAgICMgQXBwbHkgY29tcGxldGUuY2FzZXMgZnVuY3Rpb24KZGYKYGBgCgpgYGB7cn0KIyBjb252ZXJ0IHRvIG5vbWluYWwgZmFjdG9yCmRmJEdyb3VwID0gZmFjdG9yKGRmJEdyb3VwKQpkZiRwaGFzZSA9IGZhY3RvcihkZiRwaGFzZSkKYGBgCgoKYGBge3J9CmRmJGxvZ19ub3ZlbHR5IDwtIGxvZyhkZiRub3ZlbHR5KzEpIApkZiRsb2dfdXNlcl9yZXF1aXJlbWVudCA8LSBsb2coZGYkdXNlci5yZXF1aXJlbWVudCsxKQpkZiRsb2dfaW5mb3ZpcyA8LSBsb2coZGYkaW5mb3ZpcysxKQpkZiRsb2dfdG90YWwgPC0gbG9nKGRmJHRvdGFsKzEpCmRmJGxvZ19jb3VudCA8LSBsb2coZGYkY291bnQrMSkKZGYkUTdfUTdfMSA8LSBsb2coZGYkUTdfUTdfMSsxKQpkZiRRN19RN18yIDwtIGxvZyhkZiRRN19RN18yKzEpCmRmJFE4X1E4XzEgPC0gbG9nKGRmJFE4X1E4XzErMSkKZGYkUTEwIDwtIGxvZyhkZiRRMTArMSkKYGBgCgoKYGBge3J9CiMgc3RhbmRhcmRpemluZyB2YXJpYWJsZXMgZm9yIHNraWxscyBhbmQgYXNwaXJhdGlvbnMuIApjb2xzIDwtIGMoIlE3X1E3XzEiLCAiUTdfUTdfMiIsICJROF9ROF8xIiwgIlExMCIsICJsb2dfbm92ZWx0eSIsICJsb2dfdXNlcl9yZXF1aXJlbWVudCIsICJsb2dfaW5mb3ZpcyIsICJsb2dfdG90YWwiLCAibG9nX2NvdW50IikKZGZbY29sc10gPC0gc2NhbGUoZGZbY29sc10pCmRmCmBgYApgYGB7cn0KbW9kLnJlZHVjZS5ub3ZlbHR5IDwtIGxtZXIoIGxvZ19ub3ZlbHR5IH4gbG9nX2NvdW50ICsgUTdfUTdfMSArIFE3X1E3XzIgKyBROF9ROF8xICsgUTEwICsgKCAxIHwgcGhhc2UgKSwgZGF0YSA9IGRmLCBSRU1MID0gRkFMU0UpCnN1bW1hcnkobW9kLnJlZHVjZS5ub3ZlbHR5KQpBSUMobW9kLnJlZHVjZS5ub3ZlbHR5KQpCSUMobW9kLnJlZHVjZS5ub3ZlbHR5KQoKYGBgCgoKYGBge3J9Cm1vZC5mdWxsLm5vdmVsdHkgPC0gbG1lciggbG9nX25vdmVsdHkgfiBmYWN0b3IoR3JvdXApICsgbG9nX2NvdW50ICsgUTdfUTdfMSArIFE3X1E3XzIgKyBROF9ROF8xICsgUTEwICsgKDEgfCBwaGFzZSksIGRhdGEgPSBkZiwgUkVNTCA9IEZBTFNFKQpzdW1tYXJ5KG1vZC5mdWxsLm5vdmVsdHkpCkFJQyhtb2QuZnVsbC5ub3ZlbHR5KQpCSUMobW9kLmZ1bGwubm92ZWx0eSkKYW5vdmEobW9kLmZ1bGwubm92ZWx0eSkKCmBgYAoKYGBge3J9CmFub3ZhKG1vZC5yZWR1Y2Uubm92ZWx0eSwgbW9kLmZ1bGwubm92ZWx0eSkKYGBgCmBgYHtyfQpsaWJyYXJ5KEFMU00pCnN0ZXAobG0obG9nX25vdmVsdHkgfiBmYWN0b3IoR3JvdXApICsgbG9nX2NvdW50ICsgUTdfUTdfMSArIFE3X1E3XzIgKyBROF9ROF8xICsgUTEwLCBkYXRhPWRmKSwKbWV0aG9kPSJib3RoIiwgdHJhY2UgPSAxICkKYGBgCgpgYGB7cn0KbW9kLnJlZHVjZS51ciA8LSBsbShsb2dfdXNlcl9yZXF1aXJlbWVudCB+IGxvZ19jb3VudCArIFE3X1E3XzEgKyBRN19RN18yICsgUThfUThfMSArIFExMCAsIGRhdGEgPSBkZikKc3VtbWFyeShtb2QucmVkdWNlLnVyKQpBSUMobW9kLnJlZHVjZS51cikKQklDKG1vZC5yZWR1Y2UudXIpCmBgYAoKYGBge3J9Cm1vZC5mdWxsLnVyIDwtIGxtKGxvZ191c2VyX3JlcXVpcmVtZW50IH4gZmFjdG9yKEdyb3VwKSArIGxvZ19jb3VudCArIFE3X1E3XzEgKyBRN19RN18yICsgUThfUThfMSArIFExMCwgZGF0YSA9IGRmKQpzdW1tYXJ5KG1vZC5mdWxsLnVyKQpBSUMobW9kLmZ1bGwudXIpCkJJQyhtb2QuZnVsbC51cikKYGBgCmBgYHtyfQphbm92YShtb2QucmVkdWNlLnVyLCBtb2QuZnVsbC51cikKYGBgCmBgYHtyfQpsaWJyYXJ5KEFMU00pCnN0ZXAobG0obG9nX3VzZXJfcmVxdWlyZW1lbnQgfiBmYWN0b3IoR3JvdXApICsgbG9nX2NvdW50ICsgUTdfUTdfMSArIFE3X1E3XzIgKyBROF9ROF8xICsgUTEwLCBkYXRhPWRmKSwKbWV0aG9kPSJib3RoIiwgdHJhY2UgPSAxICkKYGBgCgoKCmBgYHtyfQptb2QucmVkdWNlLnZpcyA8LSBsbWVyKCBsb2dfaW5mb3ZpcyB+IGxvZ19jb3VudCArIFE3X1E3XzEgKyBRN19RN18yICsgUThfUThfMSArIFExMCArICggMSB8IHBoYXNlKSAsIGRhdGEgPSBkZiwgUkVNTCA9IEZBTFNFKQpzdW1tYXJ5KG1vZC5yZWR1Y2UudmlzKQpBSUMobW9kLnJlZHVjZS52aXMpCkJJQyhtb2QucmVkdWNlLnZpcykKYGBgCgpgYGB7cn0KbW9kLmZ1bGwudmlzIDwtIGxtZXIoIGxvZ19pbmZvdmlzIH4gZmFjdG9yKEdyb3VwKSArIGxvZ19jb3VudCArIFE3X1E3XzEgKyBRN19RN18yICsgUThfUThfMSArIFExMCArICggMSB8IHBoYXNlKSAsIGRhdGEgPSBkZiwgUkVNTCA9IEZBTFNFKQpzdW1tYXJ5KG1vZC5mdWxsLnZpcykKQUlDKG1vZC5mdWxsLnZpcykKQklDKG1vZC5mdWxsLnZpcykKYGBgCmBgYHtyfQphbm92YShtb2QucmVkdWNlLnZpcywgbW9kLmZ1bGwudmlzKQpgYGAKYGBge3J9CmxpYnJhcnkoQUxTTSkKc3RlcChsbShsb2dfaW5mb3ZpcyB+IEdyb3VwICsgbG9nX2NvdW50ICsgUTdfUTdfMSArIFE3X1E3XzIgKyBROF9ROF8xICsgUTEwLCBkYXRhPWRmKSwKbWV0aG9kPSJib3RoIiwgdHJhY2UgPSAxICkKYGBgCmBgYHtyfQptb2QucmVkdWNlLnRvdGFsIDwtIGxtKCBsb2dfdG90YWwgfiBsb2dfY291bnQgKyBRN19RN18xICsgUTdfUTdfMiArIFE4X1E4XzEgKyBRMTAgLCBkYXRhID0gZGYpCnN1bW1hcnkobW9kLnJlZHVjZS50b3RhbCkKQUlDKG1vZC5yZWR1Y2UudG90YWwpCkJJQyhtb2QucmVkdWNlLnRvdGFsKQpgYGAKCgpgYGB7cn0KbW9kLmZ1bGwudG90YWwgPC0gbG0oIGxvZ190b3RhbCB+IEdyb3VwICsgUTdfUTdfMSArIFE3X1E3XzIgKyBROF9ROF8xICsgUTEwICwgZGF0YSA9IGRmKQpzdW1tYXJ5KG1vZC5mdWxsLnRvdGFsKQpBSUMobW9kLmZ1bGwudG90YWwpCkJJQyhtb2QuZnVsbC50b3RhbCkKYGBgCmBgYHtyfQphbm92YShtb2QucmVkdWNlLnRvdGFsLCBtb2QuZnVsbC50b3RhbCkKYGBgCgpgYGB7cn0KIyBjb252ZXJ0IHRvIG5vbWluYWwgZmFjdG9yCmRmJEdyb3VwID0gZmFjdG9yKGRmJEdyb3VwKQpkZiRwaGFzZSA9IGZhY3RvcihkZiRwaGFzZSkKYGBgCgpgYGB7cn0KbGlicmFyeShwbHlyKQpkZHBseShkZiwgfiBHcm91cCAqIHBoYXNlLCBmdW5jdGlvbihkYXRhKSBzdW1tYXJ5KGRhdGEkbG9nX25vdmVsdHkpICkKZGRwbHkoZGYsIH4gR3JvdXAgKiBwaGFzZSwgc3VtbWFyaXNlLCBsb2dfbm92ZWx0eS5tZWFuPW1lYW4obG9nX25vdmVsdHkpLCBsb2dfbm92ZWx0eS5zZCA9IHNkKGxvZ19ub3ZlbHR5KSkKYGBgCmBgYHtyfQojIGhpc3RvZ3JhbXMgZm9yIHR3byBmYWN0b3JzCmJveHBsb3QobG9nX25vdmVsdHkgfiBHcm91cCAqIHBoYXNlLCBkYXRhID0gZGYsIHhsYWI9Ikdyb3VwLlBoYXNlIiwgeWxhYj0ibG9nX25vdmVsdHkiKQp3aXRoKGRmLCBpbnRlcmFjdGlvbi5wbG90KEdyb3VwLCBwaGFzZSwgbG9nX25vdmVsdHksIHlsaW09YygwLCBtYXgobG9nX25vdmVsdHkpKSkpICMgaW50ZXJhY3Rpb24gcGxvdApgYGAKCmBgYHtyfQojIGhpc3RvZ3JhbXMgZm9yIHR3byBmYWN0b3JzCmJveHBsb3QobG9nX25vdmVsdHkgfiBHcm91cCAqIHBoYXNlLCBkYXRhID0gZGYsIHhsYWI9Ikdyb3VwLlBoYXNlIiwgeWxhYj0ibG9nX3VzZXJfcmVxdWlyZW1lbnQiKQp3aXRoKGRmLCBpbnRlcmFjdGlvbi5wbG90KEdyb3VwLCBwaGFzZSwgbG9nX3VzZXJfcmVxdWlyZW1lbnQsIHlsaW09YygwLCBtYXgobG9nX3VzZXJfcmVxdWlyZW1lbnQpKSkpICMgaW50ZXJhY3Rpb24gcGxvdApgYGAKCmBgYHtyfQojIGhpc3RvZ3JhbXMgZm9yIHR3byBmYWN0b3JzCmJveHBsb3QobG9nX25vdmVsdHkgfiBHcm91cCAqIHBoYXNlLCBkYXRhID0gZGYsIHhsYWI9Ikdyb3VwLlBoYXNlIiwgeWxhYj0ibG9nX2luZm92aXMiKQp3aXRoKGRmLCBpbnRlcmFjdGlvbi5wbG90KEdyb3VwLCBwaGFzZSwgbG9nX2luZm92aXMsIHlsaW09YygwLCBtYXgobG9nX2luZm92aXMpKSkpICMgaW50ZXJhY3Rpb24gcGxvdApgYGAKCmBgYHtyfQojIGhpc3RvZ3JhbXMgZm9yIHR3byBmYWN0b3JzCmJveHBsb3QobG9nX25vdmVsdHkgfiBHcm91cCAqIHBoYXNlLCBkYXRhID0gZGYsIHhsYWI9Ikdyb3VwLlBoYXNlIiwgeWxhYj0ibG9nX3RvdGFsIikKd2l0aChkZiwgaW50ZXJhY3Rpb24ucGxvdChHcm91cCwgcGhhc2UsIGxvZ190b3RhbCwgeWxpbT1jKDAsIG1heChsb2dfdG90YWwpKSkpICMgaW50ZXJhY3Rpb24gcGxvdApgYGAKCgoKCgoKYGBge3J9Cm0gPSBsbWVyKGxvZ19ub3ZlbHR5IH4gR3JvdXAgKyAoMXxTdHVkZW50KSwgZGF0YT1kZiwgUkVNTD1GQUxTRSkKc3VtbWFyeShtKQpgYGAKYGBge3J9CnBsb3QocmVzaWQobSwgdHlwZSA9ICJwZWFyc29uIikgfiBmaXR0ZWQobSkpCnFxbm9ybShyZXNpZChtLCB0eXBlID0gInBlYXJzb24iKSkKcXFsaW5lKHJlc2lkKG0sIHR5cGUgPSAicGVhcnNvbiIpKQpgYGAKYGBge3J9CiMgbGlicmFyeSBmb3IgTE1NIHdlIHdpbGwgdXNlIG9uIHJlbGF0aW9uYWwgbG9nX25vdmVsdHkgCgpsaWJyYXJ5KGxtZTQpCmxpYnJhcnkobG1lclRlc3QpCmxpYnJhcnkoY2FyKQpgYGAKCiMgc2V0IHN1bS10by16ZXJvIGNvbnRyYXN0IGZvciBBTk9WQSBjZWxscyAKCmBgYHtyfQpjb250cmFzdHMoZGYkR3JvdXApIDw9ICJjb250ci5zdW0iCmNvbnRyYXN0cyhkZiRwaGFzZSkgPD0gImNvbnRyLnN1bSIKYGBgCgpgYGB7cn0KIyBwaGFzZSBpcyBuZXN0ZWQgd2l0aGluIGdyb3VwIApmaXQgPC0gbG0obG9nX3RvdGFsIH4gbG9nX2NvdW50ICsgUTdfUTdfMSArIFE3X1E3XzIgKyBROF9ROF8xICsgUTEwICwgZGF0YSA9IGRmKQpzdW1tYXJ5KGZpdCkKYGBgCgpgYGB7cn0KbGlicmFyeShtdWx0Y29tcCkKbGlicmFyeShsc21lYW5zKQojc3VtbWFyeShnbGh0KGZpdCwgbHNtKHBhaXJ3aXNlIH4gcm91cCAvIHBoYXNlKSksIHRlc3QgPSBhZGp1c3RlZCh0eXBlPSdob2xtJykpCmBgYAoKYGBge3J9CmZpdC5mdWxsIDwtIGxtKGxvZ190b3RhbCB+IEdyb3VwICArIGxvZ19jb3VudCArIFE3X1E3XzEgKyBRN19RN18yICsgUThfUThfMSArIFExMCwgZGF0YSA9IGRmKQpzdW1tYXJ5KGZpdC5mdWxsKQpgYGAKCmBgYHtyfQphbm92YShmaXQsIGZpdC5mdWxsKQpgYGAKYGBge3J9CiMgaGlzdG9ncmFtcyBmb3IgdHdvIGZhY3RvcnMKYm94cGxvdChsb2dfdG90YWwgfiBHcm91cCAqIHBoYXNlLCBkYXRhID0gZGYsIHhsYWI9Ikdyb3VwLlBoYXNlIiwgeWxhYj0ibG9nX3RvdGFsIikKd2l0aChkZiwgaW50ZXJhY3Rpb24ucGxvdChHcm91cCwgcGhhc2UsIGxvZ190b3RhbCwgeWxpbT1jKDAsIG1heChsb2dfdG90YWwpKSkpICMgaW50ZXJhY3Rpb24gcGxvdApgYGAKYGBge3J9CmZpdC5sbWVyIDwtIGxtZXIobG9nX3RvdGFsIH4gKDEgfCBHcm91cCkgKyAoMXwgcGhhc2U6R3JvdXApLCBkYXRhID0gZGYsIFJFTUw9IEZBTFNFKQpzdW1tYXJ5KGZpdC5sbWVyKQpgYGAKCgoKYGBge3J9CmxpYnJhcnkobG1lclRlc3QpCmZpdC5sbWVyIDwtIGxtZXIobG9nX3RvdGFsIH4gR3JvdXAgICsgbG9nX2NvdW50ICsgUTdfUTdfMSArIFE3X1E3XzIgKyBROF9ROF8xICsgUTEwICsgKCAxIHwgcGhhc2UpLCBkYXRhID0gZGYsIFJFTUw9IEZBTFNFKQpzdW1tYXJ5KGZpdC5sbWVyKQphbm92YShmaXQubG1lcikKYGBgCgpgYGB7cn0KbGlicmFyeShtdWx0Y29tcCkKc3VtbWFyeShnbGh0KGZpdC5sbWVyLCBsaW5mY3QgPSBtY3AoR3JvdXAgPSAiVHVrZXkiKSksIHRlc3QgPSBhZGp1c3RlZCgiaG9sbSIpKQpgYGAKCgpgYGB7cn0KZml0LmxtZXIucmVkdWNlZCA8LSBsbWVyKGxvZ190b3RhbCB+IGxvZ19jb3VudCArIFE3X1E3XzEgKyBRN19RN18yICsgUThfUThfMSArIFExMCArICggMSB8IHBoYXNlKSwgZGF0YSA9IGRmLCBSRU1MPUZBTFNFKQpzdW1tYXJ5KGZpdC5sbWVyLnJlZHVjZWQpCmBgYAoKYGBge3J9CmFub3ZhKGZpdC5sbWVyLnJlZHVjZWQsIGZpdC5sbWVyKQpgYGAKCgojIFVzZXIgUmVxdWlyZW1lbnQgU2NvcmUKCmBgYHtyfQojIHBoYXNlIGlzIG5lc3RlZCB3aXRoaW4gZ3JvdXAgCmZpdC5yZXF1aXJlbWVudC5mdWxsIDwtIGxtZXIobG9nX3VzZXJfcmVxdWlyZW1lbnQgfiAgZmFjdG9yKEdyb3VwKSArIGxvZ19jb3VudCArIFE3X1E3XzEgKyBRN19RN18yICsgUThfUThfMSArIFExMCArICggMSB8IHBoYXNlKSwgZGF0YSA9IGRmLCBSRU1MID0gRkFMU0UpCkFub3ZhKGZpdC5yZXF1aXJlbWVudC5mdWxsLCB0eXBlPTMsIHRlc3Quc3RhdGlzdGljcz0iRiIpCmZpdC5yZXF1aXJlbWVudC5mdWxsCmBgYAoKYGBge3J9CiMgaGlzdG9ncmFtcyBmb3IgdHdvIGZhY3RvcnMKYm94cGxvdChsb2dfdXNlcl9yZXF1aXJlbWVudCB+IEdyb3VwICogcGhhc2UsIGRhdGEgPSBkZiwgeGxhYj0iR3JvdXAuUGhhc2UiLCB5bGFiPSJsb2dfdXNlcl9yZXF1aXJlbWVudCIpCndpdGgoZGYsIGludGVyYWN0aW9uLnBsb3QoR3JvdXAsIHBoYXNlLCBsb2dfdXNlcl9yZXF1aXJlbWVudCwgeWxpbT1jKDAsIG1heChsb2dfdXNlcl9yZXF1aXJlbWVudCkpKSkgIyBpbnRlcmFjdGlvbiBwbG90CmBgYAoKCmBgYHtyfQojIHBoYXNlIGlzIG5lc3RlZCB3aXRoaW4gZ3JvdXAgCmZpdC5yZXF1aXJlbWVudCA8LSBsbWVyKGxvZ191c2VyX3JlcXVpcmVtZW50IH4gIGxvZ19jb3VudCArIFE3X1E3XzEgKyBRN19RN18yICsgUThfUThfMSArIFExMCArICAoIDEgfCBwaGFzZSksIGRhdGEgPSBkZiwgUkVNTCA9IEZBTFNFKQpBbm92YShmaXQsIHR5cGU9MywgdGVzdC5zdGF0aXN0aWNzPSJGIikKZml0LnJlcXVpcmVtZW50CmBgYAoKYGBge3J9CnBsb3QocmVzaWQobSwgdHlwZSA9ICJwZWFyc29uIikgfiBmaXR0ZWQobSkpCnFxbm9ybShyZXNpZChtLCB0eXBlID0gInBlYXJzb24iKSkKcXFsaW5lKHJlc2lkKG0sIHR5cGUgPSAicGVhcnNvbiIpKQpgYGAKCmBgYHtyfSAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAKYW5vdmEoZml0LnJlcXVpcmVtZW50LCBmaXQucmVxdWlyZW1lbnQuZnVsbCkKYGBgCiAK